Retrieval Benchmark¶
Evaluate Recall@8 on 200 dev samples.
In [1]:
try:
import finrag, jsonlines, openai
except ImportError as e:
raise RuntimeError(
"Missing dependencies – activate .venv‑notebook and run pip install -r requirements-notebook.txt"
) from e
In [2]:
import json, random, os, sys
sys.path.insert(0, os.path.abspath("src"))
from finrag.chunk_utils import build_candidate_chunks
from finrag.retriever import retrieve_evidence
from finrag.embeddings import EmbeddingStore
import matplotlib.pyplot as plt
import seaborn as sns
In [3]:
# Load only entries with QA, limit to 20 for demo speed
with open("data/dev_turn.json") as f:
raw = json.load(f)
data = [e for e in raw if "qa" in e]
random.seed(42)
samples = random.sample(data, min(20, len(data)))
In [4]:
# Precompute + cache every chunk embedding once
store = EmbeddingStore()
# build candidate chunks for each sample
all_chunks = [build_candidate_chunks(s) for s in samples]
# collect unique texts
all_texts = {c["text"] for chunks in all_chunks for c in chunks}
# batch‑fetch missing embeddings in one API call
store.get_embeddings(list(all_texts))
Out[4]: